In [22]:
this_notebook_name = "BreastSegmentationStudy-TF2"

# Update this folder name for your computer

local_data_folder = "/Usersß/Josh Ehrlich/Courses/CISC881/Project/data"
overwrite_existing_data_files = False

# All results and output will be archived with this timestamp

import datetime
save_timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
print("Save timestamp: {}".format(save_timestamp))

# Learning parameters

import numpy as np

ultrasound_size = 128
num_classes = 2
num_epochs = 200
batch_size = 64
max_learning_rate = 0.02
min_learning_rate = 0.000001
regularization_rate = 0.0001
filter_multiplier = 15
class_weights = np.array([0.15, 0.85])
learning_rate_decay = (max_learning_rate - min_learning_rate) / num_epochs

# Training data augmentation parameters

max_shift_factor = 0.12
max_rotation_angle = 10
max_zoom_factor = 1.1
min_zoom_factor = 0.8

# Evaluation parameters

acceptable_margin_mm = 1.0
mm_per_pixel = 1.0

roc_thresholds = [0.9, 0.8, 0.7, 0.65, 0.6, 0.55, 0.5, 0.45, 0.4, 0.35, 0.3, 0.25, 0.2, 0.15, 0.1,
                  0.08, 0.06, 0.04, 0.02, 0.01,
                  0.008, 0.006, 0.004, 0.002, 0.001]

'''
Provide NxM numpy array to schedule cross validation
N rounds of validation will be performed, leaving out M patients in each for validation data
All values should be valid patient IDs, or negative. Negative values are ignored.

Example 1: a leave-one-out cross validation with 3 patients would look like this:
validation_schedule_patient = np.array([[0],[1],[2]])

Example 2: a leave-two-out cross validation on 10 patients would look like this:
validation_schedule_patient = np.array([[0,1],[2,3],[4,5],[6,7],[8,9]])

Example 3: leave-one-out cross validation with 3 patients, then training on all available data (no validation):
validation_schedule_patient = np.array([[0],[1],[2],[-1]])
'''
validation_schedule_patient = np.array([[0,1,2,3], [7,11,13,18], [9,14,21,31], [19,22,29,32], [20,23,27,30]])

# Uncomment for faster debugging

roc_thresholds = [0.8, 0.6, 0.4, 0.2, 0.1, 0.01, 0.001]
num_epochs = 200
Save timestamp: 2020-12-05_10-13-47
In [23]:
import os
from random import sample
from pathlib import Path

from ipywidgets import IntProgress
from IPython.display import display, HTML

import diskcache
import girder_client
import matplotlib.pyplot as plt
import pandas as pd

import tensorflow as tf

import ultrasound_batch_generator as generator
import evaluation_metrics

from girder_apikey import girder_apikey
In [24]:
# Import aigt modules

import sys
parent_folder = os.path.dirname(os.path.abspath(os.curdir))
sys.path.append(parent_folder)

import Models.segmentation_unet as unet
import utils
In [25]:
# Creating standard folders to save data and logs

data_arrays_fullpath, notebooks_save_fullpath, results_save_fullpath, models_save_fullpath, val_data_fullpath =\
    utils.create_standard_project_folders(local_data_folder)
In [26]:
# Fetching Girder data

girder_url = "https://pocus.cs.queensu.ca/api/v1"
data_csv_file = "BreastGirder.csv"
girder_key = girder_apikey

ultrasound_arrays_by_patients, segmentation_arrays_by_patients =\
    utils.load_girder_data(data_csv_file, data_arrays_fullpath, girder_url, girder_key)
    
n_patients = len(ultrasound_arrays_by_patients)

for i in range(n_patients):
    print("Patient {} has {} ultrasounds and {} segmentations".format(
        i, ultrasound_arrays_by_patients[i].shape[0], segmentation_arrays_by_patients[i].shape[0]))
Patient 0 has 379 ultrasounds and 379 segmentations
Patient 1 has 292 ultrasounds and 292 segmentations
Patient 2 has 426 ultrasounds and 426 segmentations
Patient 3 has 418 ultrasounds and 418 segmentations
Patient 4 has 442 ultrasounds and 442 segmentations
Patient 5 has 224 ultrasounds and 224 segmentations
Patient 6 has 272 ultrasounds and 272 segmentations
Patient 7 has 288 ultrasounds and 288 segmentations
Patient 8 has 175 ultrasounds and 175 segmentations
Patient 9 has 113 ultrasounds and 113 segmentations
Patient 10 has 237 ultrasounds and 237 segmentations
Patient 11 has 180 ultrasounds and 180 segmentations
Patient 12 has 5 ultrasounds and 5 segmentations
Patient 13 has 402 ultrasounds and 402 segmentations
Patient 14 has 295 ultrasounds and 295 segmentations
Patient 15 has 331 ultrasounds and 331 segmentations
Patient 16 has 145 ultrasounds and 145 segmentations
Patient 17 has 253 ultrasounds and 253 segmentations
Patient 18 has 158 ultrasounds and 158 segmentations
Patient 19 has 187 ultrasounds and 187 segmentations
Patient 20 has 135 ultrasounds and 135 segmentations
Patient 21 has 107 ultrasounds and 107 segmentations
Patient 22 has 238 ultrasounds and 238 segmentations
Patient 23 has 226 ultrasounds and 226 segmentations
Patient 24 has 162 ultrasounds and 162 segmentations
Patient 25 has 124 ultrasounds and 124 segmentations
Patient 26 has 209 ultrasounds and 209 segmentations
Patient 27 has 123 ultrasounds and 123 segmentations
Patient 28 has 181 ultrasounds and 181 segmentations
Patient 29 has 134 ultrasounds and 134 segmentations
Patient 30 has 145 ultrasounds and 145 segmentations
Patient 31 has 133 ultrasounds and 133 segmentations
Patient 32 has 179 ultrasounds and 179 segmentations
In [27]:
# Prepare validation rounds

if np.max(np.max(validation_schedule_patient)) > (n_patients - 1):
    raise Exception("Patient ID cannot be greater than {}".format(n_patients - 1))

num_validation_rounds = len(validation_schedule_patient)
print("Planning {} rounds of validation".format(num_validation_rounds))
for i in range(num_validation_rounds):
    print("Validation on patients {} in round {}".format(validation_schedule_patient[i], i))
Planning 5 rounds of validation
Validation on patients [0 1 2 3] in round 0
Validation on patients [ 7 11 13 18] in round 1
Validation on patients [ 9 14 21 31] in round 2
Validation on patients [19 22 29 32] in round 3
Validation on patients [20 23 27 30] in round 4
In [28]:
# Print training parameters, to archive them together with the notebook output.

time_sequence_start = datetime.datetime.now()

print("Timestamp for saved files: {}".format(save_timestamp))
print("\nTraining parameters")
print("Number of epochs:    {}".format(num_epochs))
print("Step size maximum:   {}".format(max_learning_rate))
print("Step size decay:     {}".format(learning_rate_decay))
print("Batch size:          {}".format(batch_size))
print("Regularization rate: {}".format(regularization_rate))
print("")
print("Saving validation predictions in: {}".format(val_data_fullpath))
print("Saving models in:                 {}".format(models_save_fullpath))

# ROC data will be saved in these containers

val_best_metrics    = dict()
val_fuzzy_metrics   = dict()
val_aurocs          = np.zeros(num_validation_rounds)
val_best_thresholds = np.zeros(num_validation_rounds)

# Perform validation rounds

for val_round_index in range(num_validation_rounds):
    
    # Prepare data arrays
    
    train_ultrasound_data = np.zeros(
        [0,
         ultrasound_arrays_by_patients[0].shape[1],
         ultrasound_arrays_by_patients[0].shape[2],
         ultrasound_arrays_by_patients[0].shape[3]])
    
    train_segmentation_data = np.zeros(
        [0,
         segmentation_arrays_by_patients[0].shape[1],
         segmentation_arrays_by_patients[0].shape[2],
         segmentation_arrays_by_patients[0].shape[3]])
    
    val_ultrasound_data = np.zeros(
        [0,
         ultrasound_arrays_by_patients[0].shape[1],
         ultrasound_arrays_by_patients[0].shape[2],
         ultrasound_arrays_by_patients[0].shape[3]])
    
    val_segmentation_data = np.zeros(
        [0,
         segmentation_arrays_by_patients[0].shape[1],
         segmentation_arrays_by_patients[0].shape[2],
         segmentation_arrays_by_patients[0].shape[3]])
    
    for patient_index in range(n_patients):
        if patient_index not in validation_schedule_patient[val_round_index]:
            train_ultrasound_data = np.concatenate((train_ultrasound_data,
                                                    ultrasound_arrays_by_patients[patient_index]))
            train_segmentation_data = np.concatenate((train_segmentation_data,
                                                      segmentation_arrays_by_patients[patient_index]))
        else:
            val_ultrasound_data = np.concatenate((val_ultrasound_data,
                                                 ultrasound_arrays_by_patients[patient_index]))
            val_segmentation_data = np.concatenate((val_segmentation_data,
                                                   segmentation_arrays_by_patients[patient_index]))
    
    n_train = train_ultrasound_data.shape[0]
    n_val = val_ultrasound_data.shape[0]
    
    print("\n*** Leave-one-out round # {}".format(val_round_index))
    print("    Training on {} images, validating on {} images...".format(n_train, n_val))
    
    val_segmentation_data_onehot = tf.keras.utils.to_categorical(val_segmentation_data, num_classes)
    
    # Create and train model
    
    model = unet.segmentation_unet(ultrasound_size, num_classes, filter_multiplier, regularization_rate)
    
    model.compile(
        optimizer=tf.keras.optimizers.Adam(lr=max_learning_rate, decay=learning_rate_decay),
        loss=unet.weighted_categorical_crossentropy(class_weights),
        metrics=["accuracy"]
    )
    """
    #----------------------------------------------------------------------------
    import os
    import sys
    import datetime
    from random import sample
    from pathlib import Path
    import girder_client
    import matplotlib.pyplot as plt
    import pandas as pd
    import tensorflow as tf
    import numpy as np

    from Spine.ultrasound_batch_generator import train_preprocess, train_preprocess_with_maps, generate_weight_maps
    from Spine import evaluation_metrics

    from Spine.models import (
        new_unet,
        weighted_categorical_crossentropy,
        weighted_categorical_crossentropy_with_maps,
    )
    import utils


    batch_size=128, 
    num_epochs=100, 
    sagittal_only=False, 
    num_frames=1, 
    with_maps=False, 
    learning_rate=0.002,
    lr_decay=False,
    dropout=0.0,
    use_attention=True,
    num_layers=5,
    filters=16,
    use_batch_norm=True,
    load_from_save=False,

    ultrasound_size = 128
    batch_size = 128
    num_classes = 2
    min_learning_rate = 0.00001
    class_weights = np.array([0.1, 0.9])
    learning_rate_decay = (0.002-0.00001) / 100
    num_epochs = 100

    model = new_unet(
                input_size = ultrasound_size,
                num_classes=num_classes,
                num_channels=num_frames,
                use_batch_norm=use_batch_norm,
                upsample_mode="deconv",  # 'deconv' or 'simple'
                dropout=dropout,
                dropout_type="spatial",
                use_attention=use_attention,
                filters=filters,
                num_layers=num_layers,
                output_activation="softmax",
            )

    learning_rate = 0.002
    loss_func = weighted_categorical_crossentropy(class_weights)
    
    preprocess_func = train_preprocess
    print(learning_rate, learning_rate_decay)
    model.compile(
                optimizer=tf.keras.optimizers.Adam(
                    lr=learning_rate, decay=learning_rate_decay
                ),
                loss=loss_func,
                metrics=["accuracy", evaluation_metrics.jaccard_coef, evaluation_metrics.dice_coef],
            )

#-----------------------------------------------------
    """
    
    model.summary() #prints the structure of the neural network. If you change the unet,
    #print this so yuo can see it.

    training_generator = generator.UltrasoundSegmentationBatchGenerator(
        train_ultrasound_data,
        train_segmentation_data[:, :, :, 0],
        batch_size,
        (ultrasound_size, ultrasound_size),
        max_shift_factor=max_shift_factor,
        min_zoom_factor=min_zoom_factor,
        max_zoom_factor=max_zoom_factor,
        max_rotation_angle=max_rotation_angle
    )
        
    training_time_start = datetime.datetime.now()
    print("TRAINING LOG",training_generator)
    if n_val > 0:
        training_log = model.fit_generator(
            training_generator,
            validation_data=(val_ultrasound_data, val_segmentation_data_onehot),
            epochs=num_epochs,
            verbose=0)
    else:
        training_log = model.fit_generator(training_generator, epochs=num_epochs, verbose=0)
    
    training_time_stop = datetime.datetime.now()
    
    # Pring training log
    
    print("  Training time: {}".format(training_time_stop-training_time_start))
    
    # Plot training loss and metrics
    
    fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(12, 4))
    
    axes[0].plot(training_log.history['loss'], 'bo--')
    if n_val > 0:
        axes[0].plot(training_log.history['val_loss'], 'ro-')
    axes[0].set(xlabel='Epochs (n)', ylabel='Loss')
    if n_val > 0:
        axes[0].legend(['Training loss', 'Validation loss'])
    
    axes[1].plot(training_log.history['accuracy'], 'bo--')
    if n_val > 0:
        axes[1].plot(training_log.history['val_accuracy'], 'ro-')
    axes[1].set(xlabel='Epochs (n)', ylabel='Accuracy')
    if n_val > 0:
        axes[1].legend(['Training accuracy', 'Validation accuracy'])
    
    fig.tight_layout()
    
    # Archive trained model with unique filename based on notebook name and timestamp
    model_file_name = this_notebook_name + "_model-" + str(val_round_index) + "_" + save_timestamp + ".h5"
    model_fullname = os.path.join(models_save_fullpath, model_file_name)
    model.save(model_fullname)

    # Predict on validation data
    
    if n_val > 0:
        print(val_ultrasound_data.shape)
        y_pred_val  = model.predict(val_ultrasound_data)

        # Saving predictions for further evaluation

        val_prediction_filename = save_timestamp + "_prediction_" + str(val_round_index) + ".npy"
        val_prediction_fullname = os.path.join(val_data_fullpath, val_prediction_filename)
        np.save(val_prediction_fullname, y_pred_val)
        
        # Validation results

        vali_metrics_dicts, vali_best_threshold_index, vali_area = evaluation_metrics.compute_roc(
            roc_thresholds, y_pred_val, val_segmentation_data, acceptable_margin_mm, mm_per_pixel)

        val_fuzzy_metrics[val_round_index] = evaluation_metrics.compute_evaluation_metrics(
            y_pred_val, val_segmentation_data, acceptable_margin_mm, mm_per_pixel)

        val_best_metrics[val_round_index]    = vali_metrics_dicts[vali_best_threshold_index]
        val_aurocs[val_round_index]          = vali_area
        val_best_thresholds[val_round_index] = roc_thresholds[vali_best_threshold_index]
    
    # Printing total time of this validation round
    
    print("\nTotal round time:  {}".format(datetime.datetime.now() - training_time_start))
    print("")


time_sequence_stop = datetime.datetime.now()

print("\nTotal training time:   {}".format(time_sequence_stop - time_sequence_start))
Timestamp for saved files: 2020-12-05_10-13-47

Training parameters
Number of epochs:    200
Step size maximum:   0.02
Step size decay:     9.9995e-05
Batch size:          64
Regularization rate: 0.0001

Saving validation predictions in: /Usersß/Josh Ehrlich/Courses/CISC881/Project/data\PredictionsValidation
Saving models in:                 /Usersß/Josh Ehrlich/Courses/CISC881/Project/data\SavedModels

*** Leave-one-out round # 0
    Training on 5803 images, validating on 1515 images...
Model: "functional_11"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_6 (InputLayer)            [(None, 128, 128, 1) 0                                            
__________________________________________________________________________________________________
conv2d_70 (Conv2D)              (None, 64, 64, 8)    80          input_6[0][0]                    
__________________________________________________________________________________________________
max_pooling2d_35 (MaxPooling2D) (None, 64, 64, 8)    0           conv2d_70[0][0]                  
__________________________________________________________________________________________________
conv2d_71 (Conv2D)              (None, 32, 32, 16)   1168        max_pooling2d_35[0][0]           
__________________________________________________________________________________________________
max_pooling2d_36 (MaxPooling2D) (None, 32, 32, 16)   0           conv2d_71[0][0]                  
__________________________________________________________________________________________________
conv2d_72 (Conv2D)              (None, 16, 16, 32)   4640        max_pooling2d_36[0][0]           
__________________________________________________________________________________________________
max_pooling2d_37 (MaxPooling2D) (None, 16, 16, 32)   0           conv2d_72[0][0]                  
__________________________________________________________________________________________________
conv2d_73 (Conv2D)              (None, 8, 8, 64)     18496       max_pooling2d_37[0][0]           
__________________________________________________________________________________________________
max_pooling2d_38 (MaxPooling2D) (None, 8, 8, 64)     0           conv2d_73[0][0]                  
__________________________________________________________________________________________________
conv2d_74 (Conv2D)              (None, 4, 4, 128)    73856       max_pooling2d_38[0][0]           
__________________________________________________________________________________________________
max_pooling2d_39 (MaxPooling2D) (None, 4, 4, 128)    0           conv2d_74[0][0]                  
__________________________________________________________________________________________________
conv2d_75 (Conv2D)              (None, 2, 2, 256)    295168      max_pooling2d_39[0][0]           
__________________________________________________________________________________________________
max_pooling2d_40 (MaxPooling2D) (None, 2, 2, 256)    0           conv2d_75[0][0]                  
__________________________________________________________________________________________________
conv2d_76 (Conv2D)              (None, 1, 1, 512)    1180160     max_pooling2d_40[0][0]           
__________________________________________________________________________________________________
max_pooling2d_41 (MaxPooling2D) (None, 1, 1, 512)    0           conv2d_76[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_35 (UpSampling2D) (None, 2, 2, 512)    0           max_pooling2d_41[0][0]           
__________________________________________________________________________________________________
concatenate_35 (Concatenate)    (None, 2, 2, 768)    0           up_sampling2d_35[0][0]           
                                                                 max_pooling2d_40[0][0]           
__________________________________________________________________________________________________
conv2d_77 (Conv2D)              (None, 2, 2, 506)    6218234     concatenate_35[0][0]             
__________________________________________________________________________________________________
batch_normalization_30 (BatchNo (None, 2, 2, 506)    2024        conv2d_77[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_36 (UpSampling2D) (None, 4, 4, 506)    0           batch_normalization_30[0][0]     
__________________________________________________________________________________________________
concatenate_36 (Concatenate)    (None, 4, 4, 634)    0           up_sampling2d_36[0][0]           
                                                                 max_pooling2d_39[0][0]           
__________________________________________________________________________________________________
conv2d_78 (Conv2D)              (None, 4, 4, 250)    2536250     concatenate_36[0][0]             
__________________________________________________________________________________________________
batch_normalization_31 (BatchNo (None, 4, 4, 250)    1000        conv2d_78[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_37 (UpSampling2D) (None, 8, 8, 250)    0           batch_normalization_31[0][0]     
__________________________________________________________________________________________________
concatenate_37 (Concatenate)    (None, 8, 8, 314)    0           up_sampling2d_37[0][0]           
                                                                 max_pooling2d_38[0][0]           
__________________________________________________________________________________________________
conv2d_79 (Conv2D)              (None, 8, 8, 122)    613050      concatenate_37[0][0]             
__________________________________________________________________________________________________
batch_normalization_32 (BatchNo (None, 8, 8, 122)    488         conv2d_79[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_38 (UpSampling2D) (None, 16, 16, 122)  0           batch_normalization_32[0][0]     
__________________________________________________________________________________________________
concatenate_38 (Concatenate)    (None, 16, 16, 154)  0           up_sampling2d_38[0][0]           
                                                                 max_pooling2d_37[0][0]           
__________________________________________________________________________________________________
conv2d_80 (Conv2D)              (None, 16, 16, 58)   142970      concatenate_38[0][0]             
__________________________________________________________________________________________________
batch_normalization_33 (BatchNo (None, 16, 16, 58)   232         conv2d_80[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_39 (UpSampling2D) (None, 32, 32, 58)   0           batch_normalization_33[0][0]     
__________________________________________________________________________________________________
concatenate_39 (Concatenate)    (None, 32, 32, 74)   0           up_sampling2d_39[0][0]           
                                                                 max_pooling2d_36[0][0]           
__________________________________________________________________________________________________
conv2d_81 (Conv2D)              (None, 32, 32, 26)   30810       concatenate_39[0][0]             
__________________________________________________________________________________________________
batch_normalization_34 (BatchNo (None, 32, 32, 26)   104         conv2d_81[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_40 (UpSampling2D) (None, 64, 64, 26)   0           batch_normalization_34[0][0]     
__________________________________________________________________________________________________
concatenate_40 (Concatenate)    (None, 64, 64, 34)   0           up_sampling2d_40[0][0]           
                                                                 max_pooling2d_35[0][0]           
__________________________________________________________________________________________________
conv2d_82 (Conv2D)              (None, 64, 64, 10)   5450        concatenate_40[0][0]             
__________________________________________________________________________________________________
batch_normalization_35 (BatchNo (None, 64, 64, 10)   40          conv2d_82[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_41 (UpSampling2D) (None, 128, 128, 10) 0           batch_normalization_35[0][0]     
__________________________________________________________________________________________________
concatenate_41 (Concatenate)    (None, 128, 128, 11) 0           up_sampling2d_41[0][0]           
                                                                 input_6[0][0]                    
__________________________________________________________________________________________________
conv2d_83 (Conv2D)              (None, 128, 128, 2)  354         concatenate_41[0][0]             
==================================================================================================
Total params: 11,124,574
Trainable params: 11,122,630
Non-trainable params: 1,944
__________________________________________________________________________________________________
TRAINING LOG <ultrasound_batch_generator.UltrasoundSegmentationBatchGenerator object at 0x000001CF972BE2B0>
  Training time: 9:52:42.490424
(1515, 128, 128, 1)

Total round time:  9:52:55.982571


*** Leave-one-out round # 1
    Training on 6290 images, validating on 1028 images...
Model: "functional_13"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_7 (InputLayer)            [(None, 128, 128, 1) 0                                            
__________________________________________________________________________________________________
conv2d_84 (Conv2D)              (None, 64, 64, 8)    80          input_7[0][0]                    
__________________________________________________________________________________________________
max_pooling2d_42 (MaxPooling2D) (None, 64, 64, 8)    0           conv2d_84[0][0]                  
__________________________________________________________________________________________________
conv2d_85 (Conv2D)              (None, 32, 32, 16)   1168        max_pooling2d_42[0][0]           
__________________________________________________________________________________________________
max_pooling2d_43 (MaxPooling2D) (None, 32, 32, 16)   0           conv2d_85[0][0]                  
__________________________________________________________________________________________________
conv2d_86 (Conv2D)              (None, 16, 16, 32)   4640        max_pooling2d_43[0][0]           
__________________________________________________________________________________________________
max_pooling2d_44 (MaxPooling2D) (None, 16, 16, 32)   0           conv2d_86[0][0]                  
__________________________________________________________________________________________________
conv2d_87 (Conv2D)              (None, 8, 8, 64)     18496       max_pooling2d_44[0][0]           
__________________________________________________________________________________________________
max_pooling2d_45 (MaxPooling2D) (None, 8, 8, 64)     0           conv2d_87[0][0]                  
__________________________________________________________________________________________________
conv2d_88 (Conv2D)              (None, 4, 4, 128)    73856       max_pooling2d_45[0][0]           
__________________________________________________________________________________________________
max_pooling2d_46 (MaxPooling2D) (None, 4, 4, 128)    0           conv2d_88[0][0]                  
__________________________________________________________________________________________________
conv2d_89 (Conv2D)              (None, 2, 2, 256)    295168      max_pooling2d_46[0][0]           
__________________________________________________________________________________________________
max_pooling2d_47 (MaxPooling2D) (None, 2, 2, 256)    0           conv2d_89[0][0]                  
__________________________________________________________________________________________________
conv2d_90 (Conv2D)              (None, 1, 1, 512)    1180160     max_pooling2d_47[0][0]           
__________________________________________________________________________________________________
max_pooling2d_48 (MaxPooling2D) (None, 1, 1, 512)    0           conv2d_90[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_42 (UpSampling2D) (None, 2, 2, 512)    0           max_pooling2d_48[0][0]           
__________________________________________________________________________________________________
concatenate_42 (Concatenate)    (None, 2, 2, 768)    0           up_sampling2d_42[0][0]           
                                                                 max_pooling2d_47[0][0]           
__________________________________________________________________________________________________
conv2d_91 (Conv2D)              (None, 2, 2, 506)    6218234     concatenate_42[0][0]             
__________________________________________________________________________________________________
batch_normalization_36 (BatchNo (None, 2, 2, 506)    2024        conv2d_91[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_43 (UpSampling2D) (None, 4, 4, 506)    0           batch_normalization_36[0][0]     
__________________________________________________________________________________________________
concatenate_43 (Concatenate)    (None, 4, 4, 634)    0           up_sampling2d_43[0][0]           
                                                                 max_pooling2d_46[0][0]           
__________________________________________________________________________________________________
conv2d_92 (Conv2D)              (None, 4, 4, 250)    2536250     concatenate_43[0][0]             
__________________________________________________________________________________________________
batch_normalization_37 (BatchNo (None, 4, 4, 250)    1000        conv2d_92[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_44 (UpSampling2D) (None, 8, 8, 250)    0           batch_normalization_37[0][0]     
__________________________________________________________________________________________________
concatenate_44 (Concatenate)    (None, 8, 8, 314)    0           up_sampling2d_44[0][0]           
                                                                 max_pooling2d_45[0][0]           
__________________________________________________________________________________________________
conv2d_93 (Conv2D)              (None, 8, 8, 122)    613050      concatenate_44[0][0]             
__________________________________________________________________________________________________
batch_normalization_38 (BatchNo (None, 8, 8, 122)    488         conv2d_93[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_45 (UpSampling2D) (None, 16, 16, 122)  0           batch_normalization_38[0][0]     
__________________________________________________________________________________________________
concatenate_45 (Concatenate)    (None, 16, 16, 154)  0           up_sampling2d_45[0][0]           
                                                                 max_pooling2d_44[0][0]           
__________________________________________________________________________________________________
conv2d_94 (Conv2D)              (None, 16, 16, 58)   142970      concatenate_45[0][0]             
__________________________________________________________________________________________________
batch_normalization_39 (BatchNo (None, 16, 16, 58)   232         conv2d_94[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_46 (UpSampling2D) (None, 32, 32, 58)   0           batch_normalization_39[0][0]     
__________________________________________________________________________________________________
concatenate_46 (Concatenate)    (None, 32, 32, 74)   0           up_sampling2d_46[0][0]           
                                                                 max_pooling2d_43[0][0]           
__________________________________________________________________________________________________
conv2d_95 (Conv2D)              (None, 32, 32, 26)   30810       concatenate_46[0][0]             
__________________________________________________________________________________________________
batch_normalization_40 (BatchNo (None, 32, 32, 26)   104         conv2d_95[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_47 (UpSampling2D) (None, 64, 64, 26)   0           batch_normalization_40[0][0]     
__________________________________________________________________________________________________
concatenate_47 (Concatenate)    (None, 64, 64, 34)   0           up_sampling2d_47[0][0]           
                                                                 max_pooling2d_42[0][0]           
__________________________________________________________________________________________________
conv2d_96 (Conv2D)              (None, 64, 64, 10)   5450        concatenate_47[0][0]             
__________________________________________________________________________________________________
batch_normalization_41 (BatchNo (None, 64, 64, 10)   40          conv2d_96[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_48 (UpSampling2D) (None, 128, 128, 10) 0           batch_normalization_41[0][0]     
__________________________________________________________________________________________________
concatenate_48 (Concatenate)    (None, 128, 128, 11) 0           up_sampling2d_48[0][0]           
                                                                 input_7[0][0]                    
__________________________________________________________________________________________________
conv2d_97 (Conv2D)              (None, 128, 128, 2)  354         concatenate_48[0][0]             
==================================================================================================
Total params: 11,124,574
Trainable params: 11,122,630
Non-trainable params: 1,944
__________________________________________________________________________________________________
TRAINING LOG <ultrasound_batch_generator.UltrasoundSegmentationBatchGenerator object at 0x000001CF95DC51C0>
  Training time: 10:35:55.774253
(1028, 128, 128, 1)

Total round time:  10:36:05.406348


*** Leave-one-out round # 2
    Training on 6670 images, validating on 648 images...
Model: "functional_15"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_8 (InputLayer)            [(None, 128, 128, 1) 0                                            
__________________________________________________________________________________________________
conv2d_98 (Conv2D)              (None, 64, 64, 8)    80          input_8[0][0]                    
__________________________________________________________________________________________________
max_pooling2d_49 (MaxPooling2D) (None, 64, 64, 8)    0           conv2d_98[0][0]                  
__________________________________________________________________________________________________
conv2d_99 (Conv2D)              (None, 32, 32, 16)   1168        max_pooling2d_49[0][0]           
__________________________________________________________________________________________________
max_pooling2d_50 (MaxPooling2D) (None, 32, 32, 16)   0           conv2d_99[0][0]                  
__________________________________________________________________________________________________
conv2d_100 (Conv2D)             (None, 16, 16, 32)   4640        max_pooling2d_50[0][0]           
__________________________________________________________________________________________________
max_pooling2d_51 (MaxPooling2D) (None, 16, 16, 32)   0           conv2d_100[0][0]                 
__________________________________________________________________________________________________
conv2d_101 (Conv2D)             (None, 8, 8, 64)     18496       max_pooling2d_51[0][0]           
__________________________________________________________________________________________________
max_pooling2d_52 (MaxPooling2D) (None, 8, 8, 64)     0           conv2d_101[0][0]                 
__________________________________________________________________________________________________
conv2d_102 (Conv2D)             (None, 4, 4, 128)    73856       max_pooling2d_52[0][0]           
__________________________________________________________________________________________________
max_pooling2d_53 (MaxPooling2D) (None, 4, 4, 128)    0           conv2d_102[0][0]                 
__________________________________________________________________________________________________
conv2d_103 (Conv2D)             (None, 2, 2, 256)    295168      max_pooling2d_53[0][0]           
__________________________________________________________________________________________________
max_pooling2d_54 (MaxPooling2D) (None, 2, 2, 256)    0           conv2d_103[0][0]                 
__________________________________________________________________________________________________
conv2d_104 (Conv2D)             (None, 1, 1, 512)    1180160     max_pooling2d_54[0][0]           
__________________________________________________________________________________________________
max_pooling2d_55 (MaxPooling2D) (None, 1, 1, 512)    0           conv2d_104[0][0]                 
__________________________________________________________________________________________________
up_sampling2d_49 (UpSampling2D) (None, 2, 2, 512)    0           max_pooling2d_55[0][0]           
__________________________________________________________________________________________________
concatenate_49 (Concatenate)    (None, 2, 2, 768)    0           up_sampling2d_49[0][0]           
                                                                 max_pooling2d_54[0][0]           
__________________________________________________________________________________________________
conv2d_105 (Conv2D)             (None, 2, 2, 506)    6218234     concatenate_49[0][0]             
__________________________________________________________________________________________________
batch_normalization_42 (BatchNo (None, 2, 2, 506)    2024        conv2d_105[0][0]                 
__________________________________________________________________________________________________
up_sampling2d_50 (UpSampling2D) (None, 4, 4, 506)    0           batch_normalization_42[0][0]     
__________________________________________________________________________________________________
concatenate_50 (Concatenate)    (None, 4, 4, 634)    0           up_sampling2d_50[0][0]           
                                                                 max_pooling2d_53[0][0]           
__________________________________________________________________________________________________
conv2d_106 (Conv2D)             (None, 4, 4, 250)    2536250     concatenate_50[0][0]             
__________________________________________________________________________________________________
batch_normalization_43 (BatchNo (None, 4, 4, 250)    1000        conv2d_106[0][0]                 
__________________________________________________________________________________________________
up_sampling2d_51 (UpSampling2D) (None, 8, 8, 250)    0           batch_normalization_43[0][0]     
__________________________________________________________________________________________________
concatenate_51 (Concatenate)    (None, 8, 8, 314)    0           up_sampling2d_51[0][0]           
                                                                 max_pooling2d_52[0][0]           
__________________________________________________________________________________________________
conv2d_107 (Conv2D)             (None, 8, 8, 122)    613050      concatenate_51[0][0]             
__________________________________________________________________________________________________
batch_normalization_44 (BatchNo (None, 8, 8, 122)    488         conv2d_107[0][0]                 
__________________________________________________________________________________________________
up_sampling2d_52 (UpSampling2D) (None, 16, 16, 122)  0           batch_normalization_44[0][0]     
__________________________________________________________________________________________________
concatenate_52 (Concatenate)    (None, 16, 16, 154)  0           up_sampling2d_52[0][0]           
                                                                 max_pooling2d_51[0][0]           
__________________________________________________________________________________________________
conv2d_108 (Conv2D)             (None, 16, 16, 58)   142970      concatenate_52[0][0]             
__________________________________________________________________________________________________
batch_normalization_45 (BatchNo (None, 16, 16, 58)   232         conv2d_108[0][0]                 
__________________________________________________________________________________________________
up_sampling2d_53 (UpSampling2D) (None, 32, 32, 58)   0           batch_normalization_45[0][0]     
__________________________________________________________________________________________________
concatenate_53 (Concatenate)    (None, 32, 32, 74)   0           up_sampling2d_53[0][0]           
                                                                 max_pooling2d_50[0][0]           
__________________________________________________________________________________________________
conv2d_109 (Conv2D)             (None, 32, 32, 26)   30810       concatenate_53[0][0]             
__________________________________________________________________________________________________
batch_normalization_46 (BatchNo (None, 32, 32, 26)   104         conv2d_109[0][0]                 
__________________________________________________________________________________________________
up_sampling2d_54 (UpSampling2D) (None, 64, 64, 26)   0           batch_normalization_46[0][0]     
__________________________________________________________________________________________________
concatenate_54 (Concatenate)    (None, 64, 64, 34)   0           up_sampling2d_54[0][0]           
                                                                 max_pooling2d_49[0][0]           
__________________________________________________________________________________________________
conv2d_110 (Conv2D)             (None, 64, 64, 10)   5450        concatenate_54[0][0]             
__________________________________________________________________________________________________
batch_normalization_47 (BatchNo (None, 64, 64, 10)   40          conv2d_110[0][0]                 
__________________________________________________________________________________________________
up_sampling2d_55 (UpSampling2D) (None, 128, 128, 10) 0           batch_normalization_47[0][0]     
__________________________________________________________________________________________________
concatenate_55 (Concatenate)    (None, 128, 128, 11) 0           up_sampling2d_55[0][0]           
                                                                 input_8[0][0]                    
__________________________________________________________________________________________________
conv2d_111 (Conv2D)             (None, 128, 128, 2)  354         concatenate_55[0][0]             
==================================================================================================
Total params: 11,124,574
Trainable params: 11,122,630
Non-trainable params: 1,944
__________________________________________________________________________________________________
TRAINING LOG <ultrasound_batch_generator.UltrasoundSegmentationBatchGenerator object at 0x000001CF934F3BE0>
  Training time: 11:08:05.758480
(648, 128, 128, 1)

Total round time:  11:08:11.921700


*** Leave-one-out round # 3
    Training on 6580 images, validating on 738 images...
Model: "functional_17"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_9 (InputLayer)            [(None, 128, 128, 1) 0                                            
__________________________________________________________________________________________________
conv2d_112 (Conv2D)             (None, 64, 64, 8)    80          input_9[0][0]                    
__________________________________________________________________________________________________
max_pooling2d_56 (MaxPooling2D) (None, 64, 64, 8)    0           conv2d_112[0][0]                 
__________________________________________________________________________________________________
conv2d_113 (Conv2D)             (None, 32, 32, 16)   1168        max_pooling2d_56[0][0]           
__________________________________________________________________________________________________
max_pooling2d_57 (MaxPooling2D) (None, 32, 32, 16)   0           conv2d_113[0][0]                 
__________________________________________________________________________________________________
conv2d_114 (Conv2D)             (None, 16, 16, 32)   4640        max_pooling2d_57[0][0]           
__________________________________________________________________________________________________
max_pooling2d_58 (MaxPooling2D) (None, 16, 16, 32)   0           conv2d_114[0][0]                 
__________________________________________________________________________________________________
conv2d_115 (Conv2D)             (None, 8, 8, 64)     18496       max_pooling2d_58[0][0]           
__________________________________________________________________________________________________
max_pooling2d_59 (MaxPooling2D) (None, 8, 8, 64)     0           conv2d_115[0][0]                 
__________________________________________________________________________________________________
conv2d_116 (Conv2D)             (None, 4, 4, 128)    73856       max_pooling2d_59[0][0]           
__________________________________________________________________________________________________
max_pooling2d_60 (MaxPooling2D) (None, 4, 4, 128)    0           conv2d_116[0][0]                 
__________________________________________________________________________________________________
conv2d_117 (Conv2D)             (None, 2, 2, 256)    295168      max_pooling2d_60[0][0]           
__________________________________________________________________________________________________
max_pooling2d_61 (MaxPooling2D) (None, 2, 2, 256)    0           conv2d_117[0][0]                 
__________________________________________________________________________________________________
conv2d_118 (Conv2D)             (None, 1, 1, 512)    1180160     max_pooling2d_61[0][0]           
__________________________________________________________________________________________________
max_pooling2d_62 (MaxPooling2D) (None, 1, 1, 512)    0           conv2d_118[0][0]                 
__________________________________________________________________________________________________
up_sampling2d_56 (UpSampling2D) (None, 2, 2, 512)    0           max_pooling2d_62[0][0]           
__________________________________________________________________________________________________
concatenate_56 (Concatenate)    (None, 2, 2, 768)    0           up_sampling2d_56[0][0]           
                                                                 max_pooling2d_61[0][0]           
__________________________________________________________________________________________________
conv2d_119 (Conv2D)             (None, 2, 2, 506)    6218234     concatenate_56[0][0]             
__________________________________________________________________________________________________
batch_normalization_48 (BatchNo (None, 2, 2, 506)    2024        conv2d_119[0][0]                 
__________________________________________________________________________________________________
up_sampling2d_57 (UpSampling2D) (None, 4, 4, 506)    0           batch_normalization_48[0][0]     
__________________________________________________________________________________________________
concatenate_57 (Concatenate)    (None, 4, 4, 634)    0           up_sampling2d_57[0][0]           
                                                                 max_pooling2d_60[0][0]           
__________________________________________________________________________________________________
conv2d_120 (Conv2D)             (None, 4, 4, 250)    2536250     concatenate_57[0][0]             
__________________________________________________________________________________________________
batch_normalization_49 (BatchNo (None, 4, 4, 250)    1000        conv2d_120[0][0]                 
__________________________________________________________________________________________________
up_sampling2d_58 (UpSampling2D) (None, 8, 8, 250)    0           batch_normalization_49[0][0]     
__________________________________________________________________________________________________
concatenate_58 (Concatenate)    (None, 8, 8, 314)    0           up_sampling2d_58[0][0]           
                                                                 max_pooling2d_59[0][0]           
__________________________________________________________________________________________________
conv2d_121 (Conv2D)             (None, 8, 8, 122)    613050      concatenate_58[0][0]             
__________________________________________________________________________________________________
batch_normalization_50 (BatchNo (None, 8, 8, 122)    488         conv2d_121[0][0]                 
__________________________________________________________________________________________________
up_sampling2d_59 (UpSampling2D) (None, 16, 16, 122)  0           batch_normalization_50[0][0]     
__________________________________________________________________________________________________
concatenate_59 (Concatenate)    (None, 16, 16, 154)  0           up_sampling2d_59[0][0]           
                                                                 max_pooling2d_58[0][0]           
__________________________________________________________________________________________________
conv2d_122 (Conv2D)             (None, 16, 16, 58)   142970      concatenate_59[0][0]             
__________________________________________________________________________________________________
batch_normalization_51 (BatchNo (None, 16, 16, 58)   232         conv2d_122[0][0]                 
__________________________________________________________________________________________________
up_sampling2d_60 (UpSampling2D) (None, 32, 32, 58)   0           batch_normalization_51[0][0]     
__________________________________________________________________________________________________
concatenate_60 (Concatenate)    (None, 32, 32, 74)   0           up_sampling2d_60[0][0]           
                                                                 max_pooling2d_57[0][0]           
__________________________________________________________________________________________________
conv2d_123 (Conv2D)             (None, 32, 32, 26)   30810       concatenate_60[0][0]             
__________________________________________________________________________________________________
batch_normalization_52 (BatchNo (None, 32, 32, 26)   104         conv2d_123[0][0]                 
__________________________________________________________________________________________________
up_sampling2d_61 (UpSampling2D) (None, 64, 64, 26)   0           batch_normalization_52[0][0]     
__________________________________________________________________________________________________
concatenate_61 (Concatenate)    (None, 64, 64, 34)   0           up_sampling2d_61[0][0]           
                                                                 max_pooling2d_56[0][0]           
__________________________________________________________________________________________________
conv2d_124 (Conv2D)             (None, 64, 64, 10)   5450        concatenate_61[0][0]             
__________________________________________________________________________________________________
batch_normalization_53 (BatchNo (None, 64, 64, 10)   40          conv2d_124[0][0]                 
__________________________________________________________________________________________________
up_sampling2d_62 (UpSampling2D) (None, 128, 128, 10) 0           batch_normalization_53[0][0]     
__________________________________________________________________________________________________
concatenate_62 (Concatenate)    (None, 128, 128, 11) 0           up_sampling2d_62[0][0]           
                                                                 input_9[0][0]                    
__________________________________________________________________________________________________
conv2d_125 (Conv2D)             (None, 128, 128, 2)  354         concatenate_62[0][0]             
==================================================================================================
Total params: 11,124,574
Trainable params: 11,122,630
Non-trainable params: 1,944
__________________________________________________________________________________________________
TRAINING LOG <ultrasound_batch_generator.UltrasoundSegmentationBatchGenerator object at 0x000001CF913A71C0>
  Training time: 10:54:55.870503
(738, 128, 128, 1)

Total round time:  10:55:02.493379


*** Leave-one-out round # 4
    Training on 6689 images, validating on 629 images...
Model: "functional_19"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_10 (InputLayer)           [(None, 128, 128, 1) 0                                            
__________________________________________________________________________________________________
conv2d_126 (Conv2D)             (None, 64, 64, 8)    80          input_10[0][0]                   
__________________________________________________________________________________________________
max_pooling2d_63 (MaxPooling2D) (None, 64, 64, 8)    0           conv2d_126[0][0]                 
__________________________________________________________________________________________________
conv2d_127 (Conv2D)             (None, 32, 32, 16)   1168        max_pooling2d_63[0][0]           
__________________________________________________________________________________________________
max_pooling2d_64 (MaxPooling2D) (None, 32, 32, 16)   0           conv2d_127[0][0]                 
__________________________________________________________________________________________________
conv2d_128 (Conv2D)             (None, 16, 16, 32)   4640        max_pooling2d_64[0][0]           
__________________________________________________________________________________________________
max_pooling2d_65 (MaxPooling2D) (None, 16, 16, 32)   0           conv2d_128[0][0]                 
__________________________________________________________________________________________________
conv2d_129 (Conv2D)             (None, 8, 8, 64)     18496       max_pooling2d_65[0][0]           
__________________________________________________________________________________________________
max_pooling2d_66 (MaxPooling2D) (None, 8, 8, 64)     0           conv2d_129[0][0]                 
__________________________________________________________________________________________________
conv2d_130 (Conv2D)             (None, 4, 4, 128)    73856       max_pooling2d_66[0][0]           
__________________________________________________________________________________________________
max_pooling2d_67 (MaxPooling2D) (None, 4, 4, 128)    0           conv2d_130[0][0]                 
__________________________________________________________________________________________________
conv2d_131 (Conv2D)             (None, 2, 2, 256)    295168      max_pooling2d_67[0][0]           
__________________________________________________________________________________________________
max_pooling2d_68 (MaxPooling2D) (None, 2, 2, 256)    0           conv2d_131[0][0]                 
__________________________________________________________________________________________________
conv2d_132 (Conv2D)             (None, 1, 1, 512)    1180160     max_pooling2d_68[0][0]           
__________________________________________________________________________________________________
max_pooling2d_69 (MaxPooling2D) (None, 1, 1, 512)    0           conv2d_132[0][0]                 
__________________________________________________________________________________________________
up_sampling2d_63 (UpSampling2D) (None, 2, 2, 512)    0           max_pooling2d_69[0][0]           
__________________________________________________________________________________________________
concatenate_63 (Concatenate)    (None, 2, 2, 768)    0           up_sampling2d_63[0][0]           
                                                                 max_pooling2d_68[0][0]           
__________________________________________________________________________________________________
conv2d_133 (Conv2D)             (None, 2, 2, 506)    6218234     concatenate_63[0][0]             
__________________________________________________________________________________________________
batch_normalization_54 (BatchNo (None, 2, 2, 506)    2024        conv2d_133[0][0]                 
__________________________________________________________________________________________________
up_sampling2d_64 (UpSampling2D) (None, 4, 4, 506)    0           batch_normalization_54[0][0]     
__________________________________________________________________________________________________
concatenate_64 (Concatenate)    (None, 4, 4, 634)    0           up_sampling2d_64[0][0]           
                                                                 max_pooling2d_67[0][0]           
__________________________________________________________________________________________________
conv2d_134 (Conv2D)             (None, 4, 4, 250)    2536250     concatenate_64[0][0]             
__________________________________________________________________________________________________
batch_normalization_55 (BatchNo (None, 4, 4, 250)    1000        conv2d_134[0][0]                 
__________________________________________________________________________________________________
up_sampling2d_65 (UpSampling2D) (None, 8, 8, 250)    0           batch_normalization_55[0][0]     
__________________________________________________________________________________________________
concatenate_65 (Concatenate)    (None, 8, 8, 314)    0           up_sampling2d_65[0][0]           
                                                                 max_pooling2d_66[0][0]           
__________________________________________________________________________________________________
conv2d_135 (Conv2D)             (None, 8, 8, 122)    613050      concatenate_65[0][0]             
__________________________________________________________________________________________________
batch_normalization_56 (BatchNo (None, 8, 8, 122)    488         conv2d_135[0][0]                 
__________________________________________________________________________________________________
up_sampling2d_66 (UpSampling2D) (None, 16, 16, 122)  0           batch_normalization_56[0][0]     
__________________________________________________________________________________________________
concatenate_66 (Concatenate)    (None, 16, 16, 154)  0           up_sampling2d_66[0][0]           
                                                                 max_pooling2d_65[0][0]           
__________________________________________________________________________________________________
conv2d_136 (Conv2D)             (None, 16, 16, 58)   142970      concatenate_66[0][0]             
__________________________________________________________________________________________________
batch_normalization_57 (BatchNo (None, 16, 16, 58)   232         conv2d_136[0][0]                 
__________________________________________________________________________________________________
up_sampling2d_67 (UpSampling2D) (None, 32, 32, 58)   0           batch_normalization_57[0][0]     
__________________________________________________________________________________________________
concatenate_67 (Concatenate)    (None, 32, 32, 74)   0           up_sampling2d_67[0][0]           
                                                                 max_pooling2d_64[0][0]           
__________________________________________________________________________________________________
conv2d_137 (Conv2D)             (None, 32, 32, 26)   30810       concatenate_67[0][0]             
__________________________________________________________________________________________________
batch_normalization_58 (BatchNo (None, 32, 32, 26)   104         conv2d_137[0][0]                 
__________________________________________________________________________________________________
up_sampling2d_68 (UpSampling2D) (None, 64, 64, 26)   0           batch_normalization_58[0][0]     
__________________________________________________________________________________________________
concatenate_68 (Concatenate)    (None, 64, 64, 34)   0           up_sampling2d_68[0][0]           
                                                                 max_pooling2d_63[0][0]           
__________________________________________________________________________________________________
conv2d_138 (Conv2D)             (None, 64, 64, 10)   5450        concatenate_68[0][0]             
__________________________________________________________________________________________________
batch_normalization_59 (BatchNo (None, 64, 64, 10)   40          conv2d_138[0][0]                 
__________________________________________________________________________________________________
up_sampling2d_69 (UpSampling2D) (None, 128, 128, 10) 0           batch_normalization_59[0][0]     
__________________________________________________________________________________________________
concatenate_69 (Concatenate)    (None, 128, 128, 11) 0           up_sampling2d_69[0][0]           
                                                                 input_10[0][0]                   
__________________________________________________________________________________________________
conv2d_139 (Conv2D)             (None, 128, 128, 2)  354         concatenate_69[0][0]             
==================================================================================================
Total params: 11,124,574
Trainable params: 11,122,630
Non-trainable params: 1,944
__________________________________________________________________________________________________
TRAINING LOG <ultrasound_batch_generator.UltrasoundSegmentationBatchGenerator object at 0x000001CF919A87F0>
  Training time: 11:06:49.768244
(629, 128, 128, 1)

Total round time:  11:06:55.458738


Total training time:   2 days, 5:39:45.483754
In [29]:
# Arrange results in tables

metric_labels = [
    "AUROC",
    "best thresh",
    "best TP",
    "best FP",
    "best recall",
    "best precis",
    "fuzzy recall",
    "fuzzy precis",
    "fuzzy Fscore"
]

results_labels = []
  
for label in metric_labels:
    results_labels.append("Vali " + label)

results_df = pd.DataFrame(columns = results_labels)

for i in range(num_validation_rounds):
    if i in val_best_metrics.keys():
        results_df.loc[i] = [
            val_aurocs[i],
            val_best_thresholds[i],
            val_best_metrics[i][evaluation_metrics.TRUE_POSITIVE_RATE],
            val_best_metrics[i][evaluation_metrics.FALSE_POSITIVE_RATE],
            val_best_metrics[i][evaluation_metrics.RECALL],
            val_best_metrics[i][evaluation_metrics.PRECISION],
            val_fuzzy_metrics[i][evaluation_metrics.RECALL],
            val_fuzzy_metrics[i][evaluation_metrics.PRECISION],
            val_fuzzy_metrics[i][evaluation_metrics.FSCORE]
        ]

display(results_df)

print("\nAverages")

results_means_df = results_df.mean()
display(results_means_df)
Vali AUROC Vali best thresh Vali best TP Vali best FP Vali best recall Vali best precis Vali fuzzy recall Vali fuzzy precis Vali fuzzy Fscore
0 0.985481 0.010 0.980663 0.080367 0.980663 0.093274 0.746898 0.296944 0.424943
1 0.763738 0.001 0.833256 0.241580 0.833256 0.042055 0.324963 0.186591 0.237063
2 0.951957 0.100 0.937904 0.090681 0.937904 0.186334 0.832743 0.253160 0.388279
3 0.663423 0.001 0.687326 0.124233 0.687326 0.142533 0.361426 0.377303 0.369194
4 0.880972 0.001 0.924029 0.157583 0.924029 0.172170 0.475557 0.420017 0.446065
Averages
Vali AUROC           0.849114
Vali best thresh     0.022600
Vali best TP         0.872635
Vali best FP         0.138889
Vali best recall     0.872635
Vali best precis     0.127273
Vali fuzzy recall    0.548317
Vali fuzzy precis    0.306803
Vali fuzzy Fscore    0.373109
dtype: float64
In [30]:
# Print the last ROC curve for visual verification that we catch the optimal point

n = len(roc_thresholds)

roc_x = np.zeros(n)
roc_y = np.zeros(n)

for i in range(n):
    roc_x[i] = vali_metrics_dicts[i][evaluation_metrics.FALSE_POSITIVE_RATE]
    roc_y[i] = vali_metrics_dicts[i][evaluation_metrics.SENSITIVITY]
    # print("Threshold = {0:4.2f}  False pos rate = {1:4.2f}  Sensitivity = {2:4.2f}"
    #       .format(roc_thresholds[i], roc_x[i], roc_y[i]))

    
plt.figure()
plt.ylim(-0.01, 1.01)
plt.xlim(-0.01, 1.01)
plt.plot(roc_x, roc_y, color='darkred', lw=2)
Out[30]:
[<matplotlib.lines.Line2D at 0x1cf94f95670>]
In [31]:
# Save results table

csv_filename = this_notebook_name + "_" + save_timestamp + ".csv"
csv_fullname = os.path.join(results_save_fullpath, csv_filename)
results_df.to_csv(csv_fullname)

print("Results saved to: {}".format(csv_fullname))
Results saved to: /Usersß/Josh Ehrlich/Courses/CISC881/Project/data\SavedResults\BreastSegmentationStudy-TF2_2020-12-05_10-13-47.csv
In [32]:
# Display sample results

# determine the available ultrasounds in the last round of validation
last_round = validation_schedule_patient[-1]
print(val_ultrasound_data.shape)
prev_vali = 0

#20: [63, 94, 72, 121, 130, 2, 64, 111, 26, 46, 12, 67, 132, 49, 61, 90, 65, 106, 14, 100]
#23: [347, 185, 215, 228, 345, 176, 186, 335, 301, 237, 316, 288, 181, 354, 331, 261, 194, 225, 205, 153]
#27: [364, 459, 405, 464, 426, 397, 483, 416, 430, 392, 447, 401, 423, 409, 378, 458, 422, 445, 451, 379]
#30: [494, 488, 571, 626, 568, 529, 582, 486, 484, 533, 485, 574, 507, 516, 493, 499, 549, 561, 625, 523]
count = 0
patientNumber = [20, 23, 27, 30]
sample_indices = [[63, 94, 72, 121, 130, 2, 64, 111, 26, 46, 12, 67, 132, 49, 61, 90, 65, 106, 14, 100], [347, 185, 215, 228, 345, 176, 186, 335, 301, 237, 316, 288, 181, 354, 331, 261, 194, 225, 205, 153], [364, 459, 405, 464, 426, 397, 483, 416, 430, 392, 447, 401, 423, 409, 378, 458, 422, 445, 451, 379], [494, 488, 571, 626, 568, 529, 582, 486, 484, 533, 485, 574, 507, 516, 493, 499, 549, 561, 625, 523]]
for j in range(len(last_round)):
    
    num_vali =  prev_vali + ultrasound_arrays_by_patients[last_round[j]].shape[0]
    num_show = 20
    if num_vali < num_show:
        num_show = 0
    num_col = 4

    indices = [i for i in range(prev_vali, num_vali)]
    #sample_indices = sample(indices, num_show)
    print("Showing image slices for patient", patientNumber[count], ":", sample_indices[count])
    #sample_indices = [prev_vali]
    
    # update
    prev_vali = num_vali
    threshold = 0.5
    
    fig = plt.figure(figsize=(18, num_show*5))
    for i in range(num_show):
        a0 = fig.add_subplot(num_show, num_col, i*num_col+1)
        img0 = a0.imshow(np.flipud(val_ultrasound_data[sample_indices[count][i], :, :, 0].astype(np.float32)))
        a0.set_title("Patient #{} - Ultrasound #{}".format(last_round[j], sample_indices[count][i]))
        a1 = fig.add_subplot(num_show, num_col, i*num_col+2)
        img1 = a1.imshow(np.flipud(val_segmentation_data_onehot[sample_indices[count][i], :, :, 1]), vmin=0.0, vmax=1.0)
        a1.set_title("Segmentation #{}".format(sample_indices[count][i]))
        c = fig.colorbar(img1, fraction=0.046, pad=0.04)
        a2 = fig.add_subplot(num_show, num_col, i*num_col+3)
        img2 = a2.imshow(np.flipud(y_pred_val[sample_indices[count][i], :, :, 1]), vmin=0.0, vmax=1.0)
        a2.set_title("Prediction #{}".format(sample_indices[count][i]))
        c = fig.colorbar(img2, fraction=0.046, pad=0.04)
        a3 = fig.add_subplot(num_show, num_col, i*num_col+4)
        img3 = a3.imshow((np.flipud(y_pred_val[sample_indices[count][i], :, :, 1]) > threshold), vmin=0.0, vmax=1.0)
        c = fig.colorbar(img3, fraction=0.046, pad=0.04)
        a3.set_title("Thresholded #{}".format(sample_indices[count][i]))
    count = count + 1
(629, 128, 128, 1)
Showing image slices for patient 20 : [63, 94, 72, 121, 130, 2, 64, 111, 26, 46, 12, 67, 132, 49, 61, 90, 65, 106, 14, 100]
Showing image slices for patient 23 : [347, 185, 215, 228, 345, 176, 186, 335, 301, 237, 316, 288, 181, 354, 331, 261, 194, 225, 205, 153]
Showing image slices for patient 27 : [364, 459, 405, 464, 426, 397, 483, 416, 430, 392, 447, 401, 423, 409, 378, 458, 422, 445, 451, 379]
Showing image slices for patient 30 : [494, 488, 571, 626, 568, 529, 582, 486, 484, 533, 485, 574, 507, 516, 493, 499, 549, 561, 625, 523]
In [33]:
# Save notebook so all output is archived by the next cell

from IPython.display import Javascript
script = '''
require(["base/js/namespace"],function(Jupyter) {
    Jupyter.notebook.save_checkpoint();
});
'''
Javascript(script)
Out[33]:
In [ ]:
# Export HTML copy of this notebook

notebook_file_name =  "/Users/Josh Ehrlich/Courses/CISC881/Project/data/"+this_notebook_name + "_" + save_timestamp + ".html"
notebook_fullname = os.path.join(notebooks_save_fullpath, notebook_file_name)

os.system("jupyter nbconvert --to html " + this_notebook_name + " --output " + notebook_fullname)
print("Notebook saved to: {}".format(notebook_fullname))
In [ ]:
notebook_file_name
In [ ]:
print("Notebook_name", this_notebook_name)
print("fullname", notebook_fullname)
In [ ]:
# Arrange results in tables

metric_labels = [
    'overall accuracy',
    'sensitivity-wrong',
    "AUROC",
    "best thresh",
    "best TP",
    "best FP",
    "best recall",
    "best precis",
    "fuzzy recall",
    "fuzzy precis",
    "fuzzy Fscore"
]

results_labels = []
  
for label in metric_labels:
    results_labels.append("Vali " + label)

results_df = pd.DataFrame(columns = results_labels)

for i in range(num_validation_rounds):
    if i in val_best_metrics.keys():
        results_df.loc[i] = [
            val_best_metrics[i][evaluation_metrics.ACCURACY]*100,
            (val_best_metrics[i][evaluation_metrics.TRUE_POSITIVE_RATE])/(val_best_metrics[i][evaluation_metrics.TRUE_POSITIVE_RATE] + (1-val_best_metrics[i][evaluation_metrics.TRUE_POSITIVE_RATE])),
            val_aurocs[i],
            val_best_thresholds[i],
            val_best_metrics[i][evaluation_metrics.TRUE_POSITIVE_RATE],
            val_best_metrics[i][evaluation_metrics.FALSE_POSITIVE_RATE],
            val_best_metrics[i][evaluation_metrics.RECALL],
            val_best_metrics[i][evaluation_metrics.PRECISION],
            val_fuzzy_metrics[i][evaluation_metrics.RECALL],
            val_fuzzy_metrics[i][evaluation_metrics.PRECISION],
            val_fuzzy_metrics[i][evaluation_metrics.FSCORE]
        ]

display(results_df)

print("\nAverages")

results_means_df = results_df.mean()
display(results_means_df)
'''

# Arrange results in tables

#metric_labels = [
#    "Acc",
#    "AUROC",
#    "best thresh",
#    "best TP",
#    "best FP",
#    "best recall",
#    "best precis",
#    "best dice",
#    "fuzzy recall",
#    "fuzzy precis",
#    "fuzzy Fscore"
#]

metric_labels = [
    "Overall Acc",
    "Image Classification Acc",
    "Img Classification TP",
    "Img Classification FP",
    "Img Classification FN",
    "Img Classification TN",
    "Img Classification Sensitivity (TP/(TP+FN))",
    "Img Classification Specificity (TN/(TN+FP))",
    "Img Classification Precision (TP/(TP+FP))",
    "Avg Center Distance",
    "Avg Prediction Overlap",
    "Avg Missed Prediction",
    "Dice"    
]
results_labels = []

for label in metric_labels:
    results_labels.append("Test " + label)

    
results_df = pd.DataFrame(columns = results_labels)

num_models = num_validation_rounds
test_best_metrics = val_best_metrics

for i in range(num_models):
    results_df.loc[i] = [
        test_best_metrics[i][evaluation_metrics.ACCURACY]*100,
        img_ACCs[i]*100,
        img_TPs[i],
        img_FPs[i],
        img_FNs[i],
        img_TNs[i],
        img_TPs[i]/(max(1,img_TPs[i]+img_FNs[i]))*100,
        img_TNs[i]/(max(1,img_TNs[i]+img_FPs[i]))*100,
        img_TPs[i]/(max(1,img_TPs[i]+img_FPs[i]))*100,
        totDistanceErrs[i],
        totSeg[i]*100,
        totMissed[i]*100,
        #test_aurocs[i],
        #test_best_thresholds[i],
        #test_best_metrics[i][evaluation_metrics.TRUE_POSITIVE_RATE],
        #test_best_metrics[i][evaluation_metrics.FALSE_POSITIVE_RATE],
        #test_best_metrics[i][evaluation_metrics.RECALL],
        #test_best_metrics[i][evaluation_metrics.PRECISION],
        test_best_metrics[i][evaluation_metrics.DICE]*100,
        #test_fuzzy_metrics[i][evaluation_metrics.RECALL],
        #test_fuzzy_metrics[i][evaluation_metrics.PRECISION],
        #test_fuzzy_metrics[i][evaluation_metrics.FSCORE],
    ]

display(results_df)

print("\nAverages")

results_means_df = results_df.mean()
results_std_df = results_df.std()
display(results_means_df)

print("\STD")
display(results_std_df)'''